<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Getting started with rollups | Elasticsearch Guide [7.7] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch Guide [7.7]">
<link rel="up" href="xpack-rollup.html" title="Rolling up historical data">
<link rel="prev" href="rollup-api-quickref.html" title="Rollup API quick reference">
<link rel="next" href="rollup-understanding-groups.html" title="Understanding groups">
<meta name="DC.type" content="Learn/Docs/Elasticsearch/Reference/7.7">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="7.7">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch Guide [7.7]</a></span>
»
<span class="breadcrumb-link"><a href="data-rollup-transform.html">Roll up or transform your data</a></span>
»
<span class="breadcrumb-link"><a href="xpack-rollup.html">Rolling up historical data</a></span>
»
<span class="breadcrumb-node">Getting started with rollups</span>
</div>
<div class="navheader">
<span class="prev">
<a href="rollup-api-quickref.html">« Rollup API quick reference</a>
</span>
<span class="next">
<a href="rollup-understanding-groups.html">Understanding groups »</a>
</span>
</div>
<div class="section xpack">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="rollup-getting-started"></a>Getting started with rollups<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/rollup-getting-started.asciidoc">edit</a><a class="xpack_tag" href="/subscriptions"></a>
</h2>
</div></div></div>

<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>This functionality is experimental and may be changed or removed completely in a future release. Elastic will take a best effort approach to fix any issues, but experimental features are not subject to the support SLA of official GA features.</p>
</div>
</div>
<p>To use the Rollup feature, you need to create one or more "Rollup Jobs".  These jobs run continuously in the background
and rollup the index or indices that you specify, placing the rolled documents in a secondary index (also of your choosing).</p>
<p>Imagine you have a series of daily indices that hold sensor data (<code class="literal">sensor-2017-01-01</code>, <code class="literal">sensor-2017-01-02</code>, etc).  A sample document might
look like this:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">{
  "timestamp": 1516729294000,
  "temperature": 200,
  "voltage": 5.2,
  "node": "a"
}</pre>
</div>
<h4>
<a id="_creating_a_rollup_job"></a>Creating a rollup job<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/rollup-getting-started.asciidoc">edit</a>
</h4>
<p>We’d like to rollup these documents into hourly summaries, which will allow us to generate reports and dashboards with any time interval
one hour or greater.  A rollup job might look like this:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">PUT _rollup/job/sensor
{
    "index_pattern": "sensor-*",
    "rollup_index": "sensor_rollup",
    "cron": "*/30 * * * * ?",
    "page_size" :1000,
    "groups" : {
      "date_histogram": {
        "field": "timestamp",
        "fixed_interval": "60m"
      },
      "terms": {
        "fields": ["node"]
      }
    },
    "metrics": [
        {
            "field": "temperature",
            "metrics": ["min", "max", "sum"]
        },
        {
            "field": "voltage",
            "metrics": ["avg"]
        }
    ]
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1149.console"></div>
<p>We give the job the ID of "sensor" (in the url: <code class="literal">PUT _rollup/job/sensor</code>), and tell it to rollup the index pattern <code class="literal">"sensor-*"</code>.
This job will find and rollup any index that matches that pattern. Rollup summaries are then stored in the <code class="literal">"sensor_rollup"</code> index.</p>
<p>The <code class="literal">cron</code> parameter controls when and how often the job activates.  When a rollup job’s cron schedule triggers, it will begin rolling up
from where it left off after the last activation.  So if you configure the cron to run every 30 seconds, the job will process the last 30
seconds worth of data that was indexed into the <code class="literal">sensor-*</code> indices.</p>
<p>If instead the cron was configured to run once a day at midnight, the job would process the last 24 hours worth of data.  The choice is largely
preference, based on how "realtime" you want the rollups, and if you wish to process continuously or move it to off-peak hours.</p>
<p>Next, we define a set of <code class="literal">groups</code>. Essentially, we are defining the dimensions
that we wish to pivot on at a later date when querying the data. The grouping in
this job allows us to use <code class="literal">date_histogram</code> aggregations on the <code class="literal">timestamp</code> field,
rolled up at hourly intervals. It also allows us to run terms aggregations on
the <code class="literal">node</code> field.</p>
<div class="sidebar">
<div class="titlepage"><div><div>
<p class="title"><strong>Date histogram interval vs cron schedule</strong></p>
</div></div></div>
<p>You’ll note that the job’s cron is configured to run every 30 seconds, but the date_histogram is configured to
rollup at 60 minute intervals.  How do these relate?</p>
<p>The date_histogram controls the granularity of the saved data.  Data will be rolled up into hourly intervals, and you will be unable
to query with finer granularity.  The cron simply controls when the process looks for new data to rollup.  Every 30 seconds it will see
if there is a new hour’s worth of data and roll it up.  If not, the job goes back to sleep.</p>
<p>Often, it doesn’t make sense to define such a small cron (30s) on a large interval (1h), because the majority of the activations will
simply go back to sleep.  But there’s nothing wrong with it either, the job will do the right thing.</p>
</div>
<p>After defining which groups should be generated for the data, you next configure
which metrics should be collected. By default, only the <code class="literal">doc_counts</code> are
collected for each group. To make rollup useful, you will often add metrics
like averages, mins, maxes, etc. In this example, the metrics are fairly
straightforward: we want to save the min/max/sum of the <code class="literal">temperature</code>
field, and the average of the <code class="literal">voltage</code> field.</p>
<div class="sidebar">
<div class="titlepage"><div><div>
<p class="title"><strong>Averages aren’t composable?!</strong></p>
</div></div></div>
<p>If you’ve worked with rollups before, you may be cautious around averages. If an
average is saved for a 10 minute interval, it usually isn’t useful for larger
intervals. You cannot average six 10-minute averages to find a hourly average;
the average of averages is not equal to the total average.</p>
<p>For this reason, other systems tend to either omit the ability to average or
store the average at multiple intervals to support more flexible querying.</p>
<p>Instead, the data rollup features save the <code class="literal">count</code> and <code class="literal">sum</code> for the defined time
interval. This allows us to reconstruct the average at any interval greater-than
or equal to the defined interval. This gives maximum flexibility for minimal
storage costs…​ and you don’t have to worry about average accuracies (no
average of averages here!)</p>
</div>
<p>For more details about the job syntax, see <a class="xref" href="rollup-put-job.html" title="Create rollup jobs API">Create rollup jobs</a>.</p>
<p>After you execute the above command and create the job, you’ll receive the following response:</p>
<div class="pre_wrapper lang-console-result">
<pre class="programlisting prettyprint lang-console-result">{
  "acknowledged": true
}</pre>
</div>
<h4>
<a id="_starting_the_job"></a>Starting the job<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/rollup-getting-started.asciidoc">edit</a>
</h4>
<p>After the job is created, it will be sitting in an inactive state.  Jobs need to be started before they begin processing data (this allows
you to stop them later as a way to temporarily pause, without deleting the configuration).</p>
<p>To start the job, execute this command:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">POST _rollup/job/sensor/_start</pre>
</div>
<div class="console_widget" data-snippet="snippets/1150.console"></div>
<h4>
<a id="_searching_the_rolled_results"></a>Searching the rolled results<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/rollup-getting-started.asciidoc">edit</a>
</h4>
<p>After the job has run and processed some data, we can use the <a class="xref" href="rollup-search.html" title="Rollup search">Rollup search</a> endpoint to do some searching.  The Rollup feature is designed
so that you can use the same Query DSL syntax that you are accustomed to…​ it just happens to run on the rolled up data instead.</p>
<p>For example, take this query:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /sensor_rollup/_rollup_search
{
    "size": 0,
    "aggregations": {
        "max_temperature": {
            "max": {
                "field": "temperature"
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1151.console"></div>
<p>It’s a simple aggregation that calculates the maximum of the <code class="literal">temperature</code> field.  But you’ll notice that is is being sent to the <code class="literal">sensor_rollup</code>
index instead of the raw <code class="literal">sensor-*</code> indices.  And you’ll also notice that it is using the <code class="literal">_rollup_search</code> endpoint.  Otherwise the syntax
is exactly as you’d expect.</p>
<p>If you were to execute that query, you’d receive a result that looks like a normal aggregation response:</p>
<div class="pre_wrapper lang-console-result">
<pre class="programlisting prettyprint lang-console-result">{
  "took" : 102,
  "timed_out" : false,
  "terminated_early" : false,
  "_shards" : ... ,
  "hits" : {
    "total" : {
        "value": 0,
        "relation": "eq"
    },
    "max_score" : 0.0,
    "hits" : [ ]
  },
  "aggregations" : {
    "max_temperature" : {
      "value" : 202.0
    }
  }
}</pre>
</div>
<p>The only notable difference is that Rollup search results have zero <code class="literal">hits</code>, because we aren’t really searching the original, live data any
more.  Otherwise it’s identical syntax.</p>
<p>There are a few interesting takeaways here.  Firstly, even though the data was rolled up with hourly intervals and partitioned by
node name, the query we ran is just calculating the max temperature across all documents.  The <code class="literal">groups</code> that were configured in the job
are not mandatory elements of a query, they are just extra dimensions you can partition on.  Second, the request and response syntax
is nearly identical to normal DSL, making it easy to integrate into dashboards and applications.</p>
<p>Finally, we can use those grouping fields we defined to construct a more complicated query:</p>
<div class="pre_wrapper lang-console">
<pre class="programlisting prettyprint lang-console">GET /sensor_rollup/_rollup_search
{
    "size": 0,
    "aggregations": {
        "timeline": {
            "date_histogram": {
                "field": "timestamp",
                "fixed_interval": "7d"
            },
            "aggs": {
                "nodes": {
                    "terms": {
                        "field": "node"
                    },
                    "aggs": {
                        "max_temperature": {
                            "max": {
                                "field": "temperature"
                            }
                        },
                        "avg_voltage": {
                            "avg": {
                                "field": "voltage"
                            }
                        }
                    }
                }
            }
        }
    }
}</pre>
</div>
<div class="console_widget" data-snippet="snippets/1152.console"></div>
<p>Which returns a corresponding response:</p>
<div class="pre_wrapper lang-console-result">
<pre class="programlisting prettyprint lang-console-result">{
   "took" : 93,
   "timed_out" : false,
   "terminated_early" : false,
   "_shards" : ... ,
   "hits" : {
     "total" : {
        "value": 0,
        "relation": "eq"
     },
     "max_score" : 0.0,
     "hits" : [ ]
   },
   "aggregations" : {
     "timeline" : {
       "meta" : { },
       "buckets" : [
         {
           "key_as_string" : "2018-01-18T00:00:00.000Z",
           "key" : 1516233600000,
           "doc_count" : 6,
           "nodes" : {
             "doc_count_error_upper_bound" : 0,
             "sum_other_doc_count" : 0,
             "buckets" : [
               {
                 "key" : "a",
                 "doc_count" : 2,
                 "max_temperature" : {
                   "value" : 202.0
                 },
                 "avg_voltage" : {
                   "value" : 5.1499998569488525
                 }
               },
               {
                 "key" : "b",
                 "doc_count" : 2,
                 "max_temperature" : {
                   "value" : 201.0
                 },
                 "avg_voltage" : {
                   "value" : 5.700000047683716
                 }
               },
               {
                 "key" : "c",
                 "doc_count" : 2,
                 "max_temperature" : {
                   "value" : 202.0
                 },
                 "avg_voltage" : {
                   "value" : 4.099999904632568
                 }
               }
             ]
           }
         }
       ]
     }
   }
}</pre>
</div>
<p>In addition to being more complicated (date histogram and a terms aggregation, plus an additional average metric), you’ll notice
the date_histogram uses a <code class="literal">7d</code> interval instead of <code class="literal">60m</code>.</p>
<h4>
<a id="_conclusion"></a>Conclusion<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch/edit/7.7/docs/reference/rollup/rollup-getting-started.asciidoc">edit</a>
</h4>
<p>This quickstart should have provided a concise overview of the core functionality that Rollup exposes.  There are more tips and things
to consider when setting up Rollups, which you can find throughout the rest of this section.  You may also explore the <a class="xref" href="rollup-api-quickref.html" title="Rollup API quick reference">REST API</a>
for an overview of what is available.</p>
</div>
<div class="navfooter">
<span class="prev">
<a href="rollup-api-quickref.html">« Rollup API quick reference</a>
</span>
<span class="next">
<a href="rollup-understanding-groups.html">Understanding groups »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
